import json
import re
from pathlib import Path

def clean_text(text):
    """清理文本数据，移除多余空格"""
    # 使用正则表达式替换多个连续空格为一个空格
    text = re.sub(r'\s+', '', text)
    return text.strip()

def convert_to_json(input_path, output_path):
    with open(input_path, encoding='utf-8') as file:
        lines = file.readlines()

    output_data = []

    for line in lines:
        # 移除每行不必要的前缀
        line = line.strip().replace('这是一篇小红书笔记，标题是:', '')
        parts = line.split('笔记内容是:')
        if len(parts) < 2:
            continue  # 跳过没有两个部分的行

        title = clean_text(parts[0])
        content = clean_text(parts[1])

        # 检查内容是否为空，或者是否只包含空格
        if not content or content.isspace():
            continue  # 如果summary为空或只有空格，则跳过这条记录

        # 构建JSON对象
        json_object = {
            "content": title,
            "summary": content
        }
        output_data.append(json_object)

    # 将JSON数据写入文件
    with open(output_path, 'w', encoding='utf-8') as output_file:
        for item in output_data:
            output_file.write(json.dumps(item, ensure_ascii=False) + '\n')

# 使用示例
input_file_path = '/Users/kopa/Documents/dockerFactory/xhsCrawler/data/25_extracted_texts.txt'  # 修改为实际的文件路径
output_file_path = '/Users/kopa/Documents/dockerFactory/rapid_build/dev.json'  # 修改为期望的输出文件路径和名称

convert_to_json(input_file_path, output_file_path)
